#!/usr/bin/perl
#
# Countclean - count the number of occurences of a
# value in the Clean part of a database.
#
# Will give output in the format:
#
# //database:fieldname
# total:count
# empty:count
# "value":count
# "value":count
# Quotes in value are printed as \"
#
# Values containing commas are counted multiple times.
#
# Arguments: filename field...
#

use POSIX;
#use FileTable;
use Counter;

$reportlimit = 1;

while ($ARGV[0] =~ /^-/) {
    $opt = shift;
    if ($opt eq "-raw") {
	$rawmode = 1;
    } elsif ($opt eq "-n") {
	$maxrec = shift(@ARGV) + 0;
	print STDERR "Counting max $maxrec records\n";
    } elsif ($opt eq "-limit") {
	$reportlimit = shift(@ARGV) + 0;
	print STDERR "Reporting counts of $reportlimit and above\n";
    }
}

die "option -n only available if -raw used" if ($maxrec && !$rawmode);

$filename = shift;
@fields = @ARGV;
$file = FileTable::open("$filename");
die "Unable to open DB $filename\n" if !$file;

if ($rawmode) {
    processraw();
} else {
    processcooked();
}

for $field (keys(%countsx)) {
    print "//$filename:$field\n";
    for $statistic (keys(%{$countsx{$field}})) {
	print "$statistic:",$countsx{$field}{$statistic}, "\n";
    }
    $others = 0;
    for $value (sort {$counts{$field}{$b} <=> $counts{$field}{$a}}
		keys(%{$counts{$field}})) {
	if ($counts{$field}{$value} >= $reportlimit) {
	    ($valq = $value) =~ s/"/\\"/g;
	    print "\"$valq\":$counts{$field}{$value}\n";
	} else {
	    $others += $counts{$field}{$value};
	}
    }
    print "\"others\":$others\n" if $others;
}

sub processcooked()
{
    my $loops = 0;
    for my $field (@fields) {
	my $delim;
	if (($dbfield = $field) =~ s/([\/:])$//) { # trailing slash or colon means "stop here"
	    $delim = $1;
	}
	print STDERR "Processing $dbfield",
	     $delim?" with delimiter $delim":"", "\n";
	my %summary = $file->counts("$dbfield");
	for my $key (keys %summary) {
	    if ($key) {
		@values = split(/\s*,\s*/, $key);
		for $value (@values) {
		    if ($delim) {
			$value =~ s/$delim.*//;
		    }
		    $counts{$field}{$value} += $summary{$key};
		    $countsx{$field}{"values"} += $summary{$key};
		}
		$countsx{$field}{"total"} += $summary{$key};
	    } else {
		$countsx{$field}{"empty"} += $summary{$key};
	    }
	    $reccount += $summary{$key} if $loops == 0;
	}
	$loops += 1;
	    
    }
	
    print STDERR "$reccount records counted for $loops summaries\n";
}

sub processraw()
{
    while ($key = $file->each) {
	$rec = $file->get($key);
	for $field (@fields) {
	    if ($rec->{$field}) {
		@values = split(/\s*,\s*/, $rec->{$field});
		for $value (@values) {
		    ++ $counts{$field}{$value};
		    ++ $countsx{$field}{"values"};
		}
		++ $countsx{$field}{"total"};
	    } else {
		++ $countsx{$field}{"empty"};
	    }
	}
	++ $reccount;
	last if $maxrec && $reccount >= $maxrec;
	if ($reccount % 100 == 0 && -t STDERR) {
	    print STDERR "$reccount\r";
	}
    }
    print STDERR "$reccount records counted\n";
}
